package com.alex.utils.textsearch;

import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.Collection;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;

import org.apache.commons.io.FileUtils;

import com.alex.utils.http.HttpClientFactory;
import com.alex.utils.http.RequestBuilderFactory;
import com.alex.utils.search.SearchException;
import com.alex.utils.search.SearchResults;
import com.alex.utils.search.SearchResults.Document;
import com.alex.utils.search.cache.CachingSeacher;
import com.alex.utils.search.yandex.YandexXMLSearcher;

public class SiteSearch {

	private RequestBuilderFactory requestBuilderFactory;

	public static void main(String[] args) throws Exception {
		new SiteSearch().search();
	}

	private void search() throws IOException {
		String projectDir = "/home/alex/Desktop/keywords/раскрутка_сайта/";
		
		YandexXMLSearcher searcher = new YandexXMLSearcher("asobazol", "03.108020279:d51bf4fe8164fd5934c5f50ab5e1f06a");
		
		HttpClientFactory httpClientFactory = new HttpClientFactory();
		httpClientFactory.setThreadSafe(false);
		requestBuilderFactory = new RequestBuilderFactory(httpClientFactory.create());

		searcher.setRequestBuilderFactory(requestBuilderFactory);

		CachingSeacher cachingSeacher = new CachingSeacher(searcher);

		List<String> keywords = FileUtils.readLines(new File(projectDir + "parsed_keywords.txt"));

		Collection<String> sites = new LinkedHashSet<String>();
		Set<String> hosts = new HashSet<String>();
		try {
			int n = 0;
			System.out.println("Count - " + keywords.size());
			for (String keyword : keywords) {
				n++;
				System.out.println("Searching - " + keyword + ". Left - " + (keywords.size() - n));
				SearchResults searchResults = cachingSeacher.search(keyword, 0);
				for (Document document : searchResults.getDocuments()) {
					if (hosts.add(new URL(document.getUrl()).getHost())) {
						sites.add(document.getUrl());
					}
				}
				System.out.println("Found - " + searchResults.getDocuments().size());
				System.out.println();
			}
		} catch (SearchException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
		FileUtils.writeLines(new File(projectDir + "site_urls.txt"), sites);
	}
}
